{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "evaluate.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "6kQ_TbqXXsUV"
      ],
      "toc_visible": true,
      "mount_file_id": "1EJL9kFC0DlirBCpEi_IQrJJKmjA1cMXd",
      "authorship_tag": "ABX9TyMNORHJVJX/7So0UETSsN54",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "c582702a4f1d4528a76e30299f599dfb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_930c4a209a844140a8c63622eafdd8f0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_654b5194415b4b6482b067a57136b63a",
              "IPY_MODEL_d8eb5a2f52a046b0817f28b52d7daefc"
            ]
          }
        },
        "930c4a209a844140a8c63622eafdd8f0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "654b5194415b4b6482b067a57136b63a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_963e890484e3409dbc7bb888866ab429",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_60296b4a87eb4fb8b2dc8c0150ccd5d9"
          }
        },
        "d8eb5a2f52a046b0817f28b52d7daefc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_91a188a8b8234c17a085e062542921a6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1000000/? [00:01&lt;00:00, 683417.83it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_61dd7b2a28884b64a238b6c723d1fbe6"
          }
        },
        "963e890484e3409dbc7bb888866ab429": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "60296b4a87eb4fb8b2dc8c0150ccd5d9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "91a188a8b8234c17a085e062542921a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "61dd7b2a28884b64a238b6c723d1fbe6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "38742876bb934c91a82a5e48c0e395d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6568e9f2bc1a495cab978eb914c4eac4",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6f337f837a124f449a554e768812ef9c",
              "IPY_MODEL_66023a2e82ac4bfa9141948ad0080ed9"
            ]
          }
        },
        "6568e9f2bc1a495cab978eb914c4eac4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6f337f837a124f449a554e768812ef9c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_fbd9984494214436a14d802e8643d105",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a82576995afe4ef1bbb0ff49498466c4"
          }
        },
        "66023a2e82ac4bfa9141948ad0080ed9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0b27d3de82294159a6cd675777df9e89",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1000000/? [00:12&lt;00:00, 77904.06it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_967cdcb361f044fb9ce43a671ba6e578"
          }
        },
        "fbd9984494214436a14d802e8643d105": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a82576995afe4ef1bbb0ff49498466c4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0b27d3de82294159a6cd675777df9e89": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "967cdcb361f044fb9ce43a671ba6e578": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "509f5bca62e8462eaa6b76447ddd1c4a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8db4c8a02e26469aa4f3f4106ec6959e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4a9de0bf801b49418ec379da9ccd0326",
              "IPY_MODEL_0499a851aac0452dbc0160e5215a8739"
            ]
          }
        },
        "8db4c8a02e26469aa4f3f4106ec6959e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4a9de0bf801b49418ec379da9ccd0326": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f20883f8897f4ac582486ab4106b6cb9",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 100000,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 100000,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ed47142bc02c46d59475c865e5db2f71"
          }
        },
        "0499a851aac0452dbc0160e5215a8739": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a358b6fc3c3649709575c7dff023f39d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 100000/100000 [00:18&lt;00:00, 119819.47it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_465fc7dad2dd4561bd13a58578c49cc3"
          }
        },
        "f20883f8897f4ac582486ab4106b6cb9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ed47142bc02c46d59475c865e5db2f71": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a358b6fc3c3649709575c7dff023f39d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "465fc7dad2dd4561bd13a58578c49cc3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "46a77d630c064c46bdd54389cd747066": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_44733acb73fe4c73a0956e0b96bba3a3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7ebfbc2b458044298cc5a9ec145ed1b1",
              "IPY_MODEL_843531c02ec54712b7018913f1780fc8"
            ]
          }
        },
        "44733acb73fe4c73a0956e0b96bba3a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7ebfbc2b458044298cc5a9ec145ed1b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_df333f77dec041b79674dd0b39c3cabf",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 100000,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 100000,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cc6a48a04c2d4ca29d980828c310c0fc"
          }
        },
        "843531c02ec54712b7018913f1780fc8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8cc4c9f2763946db98793945386a40ff",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 100000/100000 [00:14&lt;00:00, 122491.46it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_03b884c6f5c14b5685c0076313096ed8"
          }
        },
        "df333f77dec041b79674dd0b39c3cabf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cc6a48a04c2d4ca29d980828c310c0fc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8cc4c9f2763946db98793945386a40ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "03b884c6f5c14b5685c0076313096ed8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "79e43df3e5864f30a9cc0a9c0b76d126": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d5942bc52c4d4ababa468f798876a8bc",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c8f213e3b58f47c2bae93fb89d8cfd32",
              "IPY_MODEL_e27116b1b18f4247af0b5e071a6d226e"
            ]
          }
        },
        "d5942bc52c4d4ababa468f798876a8bc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c8f213e3b58f47c2bae93fb89d8cfd32": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f9f17a90194b42eb8ae2c2f0c582b39a",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 100000,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 100000,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_863d015cc1004e53b544432f3cac298a"
          }
        },
        "e27116b1b18f4247af0b5e071a6d226e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0ed78a1bab0646e39f16f7c71a630ec5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 100000/100000 [00:11&lt;00:00, 140850.78it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_93e8fc6344b74522bec775f006286975"
          }
        },
        "f9f17a90194b42eb8ae2c2f0c582b39a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "863d015cc1004e53b544432f3cac298a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0ed78a1bab0646e39f16f7c71a630ec5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "93e8fc6344b74522bec775f006286975": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8574e7d9d8cd466681f86f62d7e6de65": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_255b58b192024725a451a056be2d72d7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6b88aab9f1a34d1f91a183e5688d72cd",
              "IPY_MODEL_526aa70f8a8d4be78645aab166c00887"
            ]
          }
        },
        "255b58b192024725a451a056be2d72d7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6b88aab9f1a34d1f91a183e5688d72cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8a934620dc94475cbb5841c392fa8537",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 100000,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 100000,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_64fe9eec0df54e3c9c270b3edb9bde05"
          }
        },
        "526aa70f8a8d4be78645aab166c00887": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_31c1355fbe1a4ef7ab6da08ac9b7051b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 100000/100000 [00:15&lt;00:00, 119141.46it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_71187f09e69a42779152b3b499e5ab7c"
          }
        },
        "8a934620dc94475cbb5841c392fa8537": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "64fe9eec0df54e3c9c270b3edb9bde05": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "31c1355fbe1a4ef7ab6da08ac9b7051b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "71187f09e69a42779152b3b499e5ab7c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ab4e7df6377548a3bdef2332fa9ca8b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_857925820b7c4064952d0565189b6a31",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2066e14366a047d39eda7ec0f1392105",
              "IPY_MODEL_0204164bfd4840f49bfb446c7f5690f9"
            ]
          }
        },
        "857925820b7c4064952d0565189b6a31": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2066e14366a047d39eda7ec0f1392105": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_75c17a4252b243d5ba6a6f613331dcd2",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 100000,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 100000,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8fbea136e5c149c7915ab08fd529e952"
          }
        },
        "0204164bfd4840f49bfb446c7f5690f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4cd0ac2896a747aabe558b076d90f3e4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 100000/100000 [00:19&lt;00:00, 106244.74it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_695272b807bc4015a051f39bf45b45a3"
          }
        },
        "75c17a4252b243d5ba6a6f613331dcd2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8fbea136e5c149c7915ab08fd529e952": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4cd0ac2896a747aabe558b076d90f3e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "695272b807bc4015a051f39bf45b45a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3440bc555d324c7bad6c2ee4c75ed674": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e0ef9ce972a64f5f8675a2ad5df891e7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a4858133405a4534bf49ef43b5aeb65a",
              "IPY_MODEL_387837219c8648f2a8b259e20c564408"
            ]
          }
        },
        "e0ef9ce972a64f5f8675a2ad5df891e7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a4858133405a4534bf49ef43b5aeb65a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_acd406ef16c7442d9cc2174cd4b81974",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 100000,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 100000,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b1ed323d762a4b77976937c517693cb0"
          }
        },
        "387837219c8648f2a8b259e20c564408": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2e43010cbcec47d1960c72918397a81e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 100000/100000 [00:11&lt;00:00, 138632.67it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_02a8a6a7fb1c4fedbd0878ab0a0276bb"
          }
        },
        "acd406ef16c7442d9cc2174cd4b81974": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b1ed323d762a4b77976937c517693cb0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2e43010cbcec47d1960c72918397a81e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "02a8a6a7fb1c4fedbd0878ab0a0276bb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7b49eb47c1754b4da877866c19059834": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_63736e9e37044ffbb8e531959b2de060",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e20b17c5669048698267cf341add521f",
              "IPY_MODEL_ef9acfcd8c3c40a5924fbf2691dadcf6"
            ]
          }
        },
        "63736e9e37044ffbb8e531959b2de060": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e20b17c5669048698267cf341add521f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3ff3243240b74103b4412da37a0badc3",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 100000,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 100000,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9aa6c0b352d24cce8a110ce167618bf4"
          }
        },
        "ef9acfcd8c3c40a5924fbf2691dadcf6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_95798d01a09541218c176006cff5e0fa",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 100000/100000 [00:17&lt;00:00, 128247.89it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_aae96f1bfb3a4a9087b77aecfc9747e1"
          }
        },
        "3ff3243240b74103b4412da37a0badc3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9aa6c0b352d24cce8a110ce167618bf4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "95798d01a09541218c176006cff5e0fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "aae96f1bfb3a4a9087b77aecfc9747e1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "90023547667c4f3f80370d7e2baac5ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_111c6d3fdcc7416c8f4ce17ccb59c8e8",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a65bc11ad7104be28c7ce3c5cc4c0766",
              "IPY_MODEL_b3cb537f4ad745cbb3cf1ddc6c8974bf"
            ]
          }
        },
        "111c6d3fdcc7416c8f4ce17ccb59c8e8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a65bc11ad7104be28c7ce3c5cc4c0766": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_28c1e2fdcf404b04ab0d43644ae595d2",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 100000,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 100000,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_43c9061a42784dda93d30f1e7b723ec7"
          }
        },
        "b3cb537f4ad745cbb3cf1ddc6c8974bf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0ed0a5b4b7ae4bd08ee83afe5393095d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 100000/100000 [00:18&lt;00:00, 128377.78it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_44ed94b24c33412da95e20c563f39671"
          }
        },
        "28c1e2fdcf404b04ab0d43644ae595d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "43c9061a42784dda93d30f1e7b723ec7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0ed0a5b4b7ae4bd08ee83afe5393095d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "44ed94b24c33412da95e20c563f39671": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1a41adb523f941dfb3140403b9a9f813": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2da0f6faf5cc429eb699c89b8465d358",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_99102b4d5c3f41cebcc66b2ca53a65d6",
              "IPY_MODEL_68c8c0bff4b145f485c61586b1ab0943"
            ]
          }
        },
        "2da0f6faf5cc429eb699c89b8465d358": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "99102b4d5c3f41cebcc66b2ca53a65d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0078481b48ca46cea0ceda01a917ce33",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 100000,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 100000,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0f0be53f3c0344d785257844220984f2"
          }
        },
        "68c8c0bff4b145f485c61586b1ab0943": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_23faa4279c654f7599e2743cf635f9f3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 100000/100000 [00:19&lt;00:00, 123299.40it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8dfbae54e84c4323aceb94315e8e3fc5"
          }
        },
        "0078481b48ca46cea0ceda01a917ce33": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0f0be53f3c0344d785257844220984f2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "23faa4279c654f7599e2743cf635f9f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8dfbae54e84c4323aceb94315e8e3fc5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cb5b69e553e844c18e18a8a90c3fdf91": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1d598b2ad7f8402d8d1649d97c639949",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4fdbb0caa5794801ae4c3ee3219f6da6",
              "IPY_MODEL_c94efc671d094f8fb788b42b570fc278"
            ]
          }
        },
        "1d598b2ad7f8402d8d1649d97c639949": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4fdbb0caa5794801ae4c3ee3219f6da6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f3572c0d3797458fa4946662244e6407",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 100000,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 100000,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4c7b4d004967432abc757360124b0f0d"
          }
        },
        "c94efc671d094f8fb788b42b570fc278": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b1b47a50e27144e2b018f8ddd27d57db",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 100000/100000 [00:11&lt;00:00, 110581.03it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5c763d671d474583bb46635e0254cdab"
          }
        },
        "f3572c0d3797458fa4946662244e6407": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4c7b4d004967432abc757360124b0f0d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b1b47a50e27144e2b018f8ddd27d57db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5c763d671d474583bb46635e0254cdab": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/bminixhofer/nnsplit/blob/master/train/evaluate.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6kQ_TbqXXsUV",
        "colab_type": "text"
      },
      "source": [
        "# Setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8ttjOr09VTEL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        },
        "outputId": "d493d64d-4782-4800-f932-6e1a51e524f9"
      },
      "source": [
        "!git clone https://github.com/bminixhofer/nnsplit"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'nnsplit'...\n",
            "remote: Enumerating objects: 142, done.\u001b[K\n",
            "remote: Counting objects: 100% (142/142), done.\u001b[K\n",
            "remote: Compressing objects: 100% (109/109), done.\u001b[K\n",
            "remote: Total 1530 (delta 61), reused 90 (delta 33), pack-reused 1388\u001b[K\n",
            "Receiving objects: 100% (1530/1530), 12.81 MiB | 28.45 MiB/s, done.\n",
            "Resolving deltas: 100% (784/784), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6M_llCmiXUoe",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "47b86dd7-de0d-4433-90d6-c70405f938b7"
      },
      "source": [
        "!pip install -r nnsplit/train/requirements.txt"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: absl-py==0.9.0 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 1)) (0.9.0)\n",
            "Collecting appdirs==1.4.4\n",
            "  Downloading https://files.pythonhosted.org/packages/3b/00/2344469e2084fb287c2e0b57b72910309874c3245463acd6cf5e3db69324/appdirs-1.4.4-py2.py3-none-any.whl\n",
            "Requirement already satisfied: attrs==19.3.0 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 3)) (19.3.0)\n",
            "Collecting black==19.10b0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/fd/bb/ad34bbc93d1bea3de086d7c59e528d4a503ac8fe318bd1fa48605584c3d2/black-19.10b0-py36-none-any.whl (97kB)\n",
            "\u001b[K     |████████████████████████████████| 102kB 2.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: blis==0.4.1 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 5)) (0.4.1)\n",
            "Collecting cachetools==4.1.0\n",
            "  Downloading https://files.pythonhosted.org/packages/b3/59/524ffb454d05001e2be74c14745b485681c6ed5f2e625f71d135704c0909/cachetools-4.1.0-py3-none-any.whl\n",
            "Requirement already satisfied: catalogue==1.0.0 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 7)) (1.0.0)\n",
            "Requirement already satisfied: certifi==2020.6.20 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 8)) (2020.6.20)\n",
            "Requirement already satisfied: chardet==3.0.4 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 9)) (3.0.4)\n",
            "Requirement already satisfied: click==7.1.2 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 10)) (7.1.2)\n",
            "Collecting configparser==5.0.0\n",
            "  Downloading https://files.pythonhosted.org/packages/4b/6b/01baa293090240cf0562cc5eccb69c6f5006282127f2b846fad011305c79/configparser-5.0.0-py3-none-any.whl\n",
            "Requirement already satisfied: cymem==2.0.3 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 12)) (2.0.3)\n",
            "Collecting diskcache==4.1.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ee/cc/d992e1d886d5ce15d2622c2e89b6de52b48312c6f05e34b7ee881b4ccb02/diskcache-4.1.0-py2.py3-none-any.whl (44kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 5.6MB/s \n",
            "\u001b[?25hCollecting docker-pycreds==0.4.0\n",
            "  Downloading https://files.pythonhosted.org/packages/f5/e8/f6bd1eee09314e7e6dee49cbe2c5e22314ccdb38db16c9fc72d2fa80d054/docker_pycreds-0.4.0-py2.py3-none-any.whl\n",
            "Collecting flake8==3.8.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/6c/20/6326a9a0c6f0527612bae748c4c03df5cd69cf06dfb2cf59d85c6e165a6a/flake8-3.8.3-py2.py3-none-any.whl (72kB)\n",
            "\u001b[K     |████████████████████████████████| 81kB 8.2MB/s \n",
            "\u001b[?25hCollecting future==0.18.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/45/0b/38b06fd9b92dc2b68d58b75f900e97884c45bedd2ff83203d933cf5851c9/future-0.18.2.tar.gz (829kB)\n",
            "\u001b[K     |████████████████████████████████| 829kB 27.0MB/s \n",
            "\u001b[?25hCollecting gitdb==4.0.5\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/48/11/d1800bca0a3bae820b84b7d813ad1eff15a48a64caea9c823fc8c1b119e8/gitdb-4.0.5-py3-none-any.whl (63kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 7.5MB/s \n",
            "\u001b[?25hCollecting GitPython==3.1.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/8c/f9/c315aa88e51fabdc08e91b333cfefb255aff04a2ee96d632c32cb19180c9/GitPython-3.1.3-py3-none-any.whl (451kB)\n",
            "\u001b[K     |████████████████████████████████| 460kB 43.4MB/s \n",
            "\u001b[?25hCollecting google-auth==1.18.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/21/57/d706964a7e4056f3f2244e16705388c11631fbb53d3e2d2a2d0fbc24d470/google_auth-1.18.0-py2.py3-none-any.whl (90kB)\n",
            "\u001b[K     |████████████████████████████████| 92kB 9.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: google-auth-oauthlib==0.4.1 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 20)) (0.4.1)\n",
            "Collecting gql==0.2.0\n",
            "  Downloading https://files.pythonhosted.org/packages/c4/6f/cf9a3056045518f06184e804bae89390eb706168349daa9dff8ac609962a/gql-0.2.0.tar.gz\n",
            "Collecting graphql-core==1.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/b0/89/00ad5e07524d8c523b14d70c685e0299a8b0de6d0727e368c41b89b7ed0b/graphql-core-1.1.tar.gz (70kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 7.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: grpcio==1.30.0 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 23)) (1.30.0)\n",
            "Collecting idna==2.9\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/89/e3/afebe61c546d18fb1709a61bee788254b40e736cff7271c7de5de2dc4128/idna-2.9-py2.py3-none-any.whl (58kB)\n",
            "\u001b[K     |████████████████████████████████| 61kB 6.7MB/s \n",
            "\u001b[?25hCollecting importlib-metadata==1.6.1\n",
            "  Downloading https://files.pythonhosted.org/packages/98/13/a1d703ec396ade42c1d33df0e1cb691a28b7c08b336a5683912c87e04cd7/importlib_metadata-1.6.1-py2.py3-none-any.whl\n",
            "Collecting joblib==0.15.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/b8/a6/d1a816b89aa1e9e96bcb298eb1ee1854f21662ebc6d55ffa3d7b3b50122b/joblib-0.15.1-py3-none-any.whl (298kB)\n",
            "\u001b[K     |████████████████████████████████| 307kB 44.1MB/s \n",
            "\u001b[?25hCollecting lxml==4.5.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/55/6f/c87dffdd88a54dd26a3a9fef1d14b6384a9933c455c54ce3ca7d64a84c88/lxml-4.5.1-cp36-cp36m-manylinux1_x86_64.whl (5.5MB)\n",
            "\u001b[K     |████████████████████████████████| 5.5MB 42.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: Markdown==3.2.2 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 28)) (3.2.2)\n",
            "Collecting mccabe==0.6.1\n",
            "  Downloading https://files.pythonhosted.org/packages/87/89/479dc97e18549e21354893e4ee4ef36db1d237534982482c3681ee6e7b57/mccabe-0.6.1-py2.py3-none-any.whl\n",
            "Requirement already satisfied: murmurhash==1.0.2 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 30)) (1.0.2)\n",
            "Collecting numpy==1.19.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/93/0b/71ae818646c1a80fbe6776d41f480649523ed31243f1f34d9d7e41d70195/numpy-1.19.0-cp36-cp36m-manylinux2010_x86_64.whl (14.6MB)\n",
            "\u001b[K     |████████████████████████████████| 14.6MB 300kB/s \n",
            "\u001b[?25hRequirement already satisfied: nvidia-ml-py3==7.352.0 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 32)) (7.352.0)\n",
            "Requirement already satisfied: oauthlib==3.1.0 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 33)) (3.1.0)\n",
            "Requirement already satisfied: pandas==1.0.5 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 34)) (1.0.5)\n",
            "Collecting pathspec==0.8.0\n",
            "  Downloading https://files.pythonhosted.org/packages/5d/d0/887c58853bd4b6ffc7aa9cdba4fc57d7b979b45888a6bd47e4568e1cf868/pathspec-0.8.0-py2.py3-none-any.whl\n",
            "Collecting pathtools==0.1.2\n",
            "  Downloading https://files.pythonhosted.org/packages/e7/7f/470d6fcdf23f9f3518f6b0b76be9df16dcc8630ad409947f8be2eb0ed13a/pathtools-0.1.2.tar.gz\n",
            "Requirement already satisfied: plac==1.1.3 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 37)) (1.1.3)\n",
            "Requirement already satisfied: preshed==3.0.2 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 38)) (3.0.2)\n",
            "Requirement already satisfied: promise==2.3 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 39)) (2.3)\n",
            "Requirement already satisfied: protobuf==3.12.2 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 40)) (3.12.2)\n",
            "Collecting psutil==5.7.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c4/b8/3512f0e93e0db23a71d82485ba256071ebef99b227351f0f5540f744af41/psutil-5.7.0.tar.gz (449kB)\n",
            "\u001b[K     |████████████████████████████████| 450kB 43.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyasn1==0.4.8 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 42)) (0.4.8)\n",
            "Requirement already satisfied: pyasn1-modules==0.2.8 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 43)) (0.2.8)\n",
            "Collecting pycodestyle==2.6.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/10/5b/88879fb861ab79aef45c7e199cae3ef7af487b5603dcb363517a50602dd7/pycodestyle-2.6.0-py2.py3-none-any.whl (41kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 5.6MB/s \n",
            "\u001b[?25hCollecting pyflakes==2.2.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/69/5b/fd01b0c696f2f9a6d2c839883b642493b431f28fa32b29abc465ef675473/pyflakes-2.2.0-py2.py3-none-any.whl (66kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 6.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: python-dateutil==2.8.1 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 46)) (2.8.1)\n",
            "Collecting pytorch-lightning==0.8.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/68/8e/d1bb6f3696aaed40bf8263c0d9be95593dbc6a63921cca68f0e7f60e7893/pytorch_lightning-0.8.1-py3-none-any.whl (293kB)\n",
            "\u001b[K     |████████████████████████████████| 296kB 49.0MB/s \n",
            "\u001b[?25hCollecting pytz==2020.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/4f/a4/879454d49688e2fad93e59d7d4efda580b783c745fd2ec2a3adf87b0808d/pytz-2020.1-py2.py3-none-any.whl (510kB)\n",
            "\u001b[K     |████████████████████████████████| 512kB 46.9MB/s \n",
            "\u001b[?25hCollecting PyYAML==5.3.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/64/c2/b80047c7ac2478f9501676c988a5411ed5572f35d1beff9cae07d321512c/PyYAML-5.3.1.tar.gz (269kB)\n",
            "\u001b[K     |████████████████████████████████| 276kB 46.8MB/s \n",
            "\u001b[?25hCollecting regex==2020.6.8\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/1a/a1/6d8fdf4a20ffbbf2bd6003dff47a0628b9e6a4b840c421b0dec27da9376e/regex-2020.6.8-cp36-cp36m-manylinux2010_x86_64.whl (660kB)\n",
            "\u001b[K     |████████████████████████████████| 665kB 20.6MB/s \n",
            "\u001b[?25hCollecting requests==2.24.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/45/1e/0c169c6a5381e241ba7404532c16a21d86ab872c9bed8bdcd4c423954103/requests-2.24.0-py2.py3-none-any.whl (61kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 3.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests-oauthlib==1.3.0 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 52)) (1.3.0)\n",
            "Requirement already satisfied: rsa==4.6 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 53)) (4.6)\n",
            "Collecting scikit-learn==0.23.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d9/3a/eb8d7bbe28f4787d140bb9df685b7d5bf6115c0e2a969def4027144e98b6/scikit_learn-0.23.1-cp36-cp36m-manylinux1_x86_64.whl (6.8MB)\n",
            "\u001b[K     |████████████████████████████████| 6.9MB 32.2MB/s \n",
            "\u001b[?25hCollecting scipy==1.5.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/06/20/d4410683e4d416a11ebc60138f6d925f571ffcfcc3794baf78fff982c98d/scipy-1.5.0-cp36-cp36m-manylinux1_x86_64.whl (25.9MB)\n",
            "\u001b[K     |████████████████████████████████| 25.9MB 1.3MB/s \n",
            "\u001b[?25hCollecting sentry-sdk==0.15.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/2f/6b/939519d77c95a9b2c85b771e9dccbf9e69cb90016c7cd63887c26400dd7a/sentry_sdk-0.15.1-py2.py3-none-any.whl (105kB)\n",
            "\u001b[K     |████████████████████████████████| 112kB 45.7MB/s \n",
            "\u001b[?25hCollecting shortuuid==1.0.1\n",
            "  Downloading https://files.pythonhosted.org/packages/25/a6/2ecc1daa6a304e7f1b216f0896b26156b78e7c38e1211e9b798b4716c53d/shortuuid-1.0.1-py3-none-any.whl\n",
            "Collecting six==1.15.0\n",
            "  Downloading https://files.pythonhosted.org/packages/ee/ff/48bde5c0f013094d729fe4b0316ba2a24774b3ff1c52d924a8a4cb04078a/six-1.15.0-py2.py3-none-any.whl\n",
            "Collecting smmap==3.0.4\n",
            "  Downloading https://files.pythonhosted.org/packages/b0/9a/4d409a6234eb940e6a78dfdfc66156e7522262f5f2fecca07dc55915952d/smmap-3.0.4-py2.py3-none-any.whl\n",
            "Collecting spacy==2.3.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/31/c7/e66e2af1cfa418c3a3917c116c4e00ccffa546f18f59e6acd7953d833c5c/spacy-2.3.0-cp36-cp36m-manylinux1_x86_64.whl (10.0MB)\n",
            "\u001b[K     |████████████████████████████████| 10.0MB 24.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: srsly==1.0.2 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 61)) (1.0.2)\n",
            "Collecting subprocess32==3.5.4\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/32/c8/564be4d12629b912ea431f1a50eb8b3b9d00f1a0b1ceff17f266be190007/subprocess32-3.5.4.tar.gz (97kB)\n",
            "\u001b[K     |████████████████████████████████| 102kB 10.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: tensorboard==2.2.2 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 63)) (2.2.2)\n",
            "Collecting tensorboard-plugin-wit==1.6.0.post3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/51/cd/a0c1f9e4582ea64dddf76c1b808b318d01e3b858a51c715bffab1016ecc7/tensorboard_plugin_wit-1.6.0.post3-py3-none-any.whl (777kB)\n",
            "\u001b[K     |████████████████████████████████| 778kB 38.6MB/s \n",
            "\u001b[?25hCollecting thinc==7.4.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/10/ae/ef3ae5e93639c0ef8e3eb32e3c18341e511b3c515fcfc603f4b808087651/thinc-7.4.1-cp36-cp36m-manylinux1_x86_64.whl (2.1MB)\n",
            "\u001b[K     |████████████████████████████████| 2.1MB 34.8MB/s \n",
            "\u001b[?25hCollecting threadpoolctl==2.1.0\n",
            "  Downloading https://files.pythonhosted.org/packages/f7/12/ec3f2e203afa394a149911729357aa48affc59c20e2c1c8297a60f33f133/threadpoolctl-2.1.0-py3-none-any.whl\n",
            "Collecting toml==0.10.1\n",
            "  Downloading https://files.pythonhosted.org/packages/9f/e1/1b40b80f2e1663a6b9f497123c11d7d988c0919abbf3c3f2688e448c5363/toml-0.10.1-py2.py3-none-any.whl\n",
            "Requirement already satisfied: torch==1.5.1 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 68)) (1.5.1+cu101)\n",
            "Collecting tqdm==4.46.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f3/76/4697ce203a3d42b2ead61127b35e5fcc26bba9a35c03b32a2bd342a4c869/tqdm-4.46.1-py2.py3-none-any.whl (63kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 7.7MB/s \n",
            "\u001b[?25hCollecting typed-ast==1.4.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/90/ed/5459080d95eb87a02fe860d447197be63b6e2b5e9ff73c2b0a85622994f4/typed_ast-1.4.1-cp36-cp36m-manylinux1_x86_64.whl (737kB)\n",
            "\u001b[K     |████████████████████████████████| 747kB 37.2MB/s \n",
            "\u001b[?25hCollecting urllib3==1.25.9\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e1/e5/df302e8017440f111c11cc41a6b432838672f5a70aa29227bf58149dc72f/urllib3-1.25.9-py2.py3-none-any.whl (126kB)\n",
            "\u001b[K     |████████████████████████████████| 133kB 43.5MB/s \n",
            "\u001b[?25hCollecting wandb==0.9.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/4b/85/ee3bec131a1078f0450140abc6c3f17177e832109c44912dca2a82e9f7d9/wandb-0.9.1-py2.py3-none-any.whl (1.4MB)\n",
            "\u001b[K     |████████████████████████████████| 1.4MB 36.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: wasabi==0.7.0 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 73)) (0.7.0)\n",
            "Collecting watchdog==0.10.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/73/c3/ed6d992006837e011baca89476a4bbffb0a91602432f73bd4473816c76e2/watchdog-0.10.2.tar.gz (95kB)\n",
            "\u001b[K     |████████████████████████████████| 102kB 10.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: Werkzeug==1.0.1 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 75)) (1.0.1)\n",
            "Requirement already satisfied: zipp==3.1.0 in /usr/local/lib/python3.6/dist-packages (from -r nnsplit/train/requirements.txt (line 76)) (3.1.0)\n",
            "Collecting somajo==2.1.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c8/5b/39e89ed5261ad596d3c2893ad6a31ba79f29976737df83b8fcb6c9df25d3/SoMaJo-2.1.1-py3-none-any.whl (89kB)\n",
            "\u001b[K     |████████████████████████████████| 92kB 9.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: setuptools>=40.3.0 in /usr/local/lib/python3.6/dist-packages (from google-auth==1.18.0->-r nnsplit/train/requirements.txt (line 19)) (49.1.0)\n",
            "Requirement already satisfied: wheel>=0.26; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from tensorboard==2.2.2->-r nnsplit/train/requirements.txt (line 63)) (0.34.2)\n",
            "Building wheels for collected packages: future, gql, graphql-core, pathtools, psutil, PyYAML, subprocess32, watchdog\n",
            "  Building wheel for future (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for future: filename=future-0.18.2-cp36-none-any.whl size=491057 sha256=3c9cbf9c302e1a50f57e81435d2b81079f0873ccfde637aed6fae3343d1aa0d0\n",
            "  Stored in directory: /root/.cache/pip/wheels/8b/99/a0/81daf51dcd359a9377b110a8a886b3895921802d2fc1b2397e\n",
            "  Building wheel for gql (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for gql: filename=gql-0.2.0-cp36-none-any.whl size=7630 sha256=a9998e7e30080ecb539ac1716d81848e8f4aca74ffc654b103c84885be986fc1\n",
            "  Stored in directory: /root/.cache/pip/wheels/ce/0e/7b/58a8a5268655b3ad74feef5aa97946f0addafb3cbb6bd2da23\n",
            "  Building wheel for graphql-core (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for graphql-core: filename=graphql_core-1.1-cp36-none-any.whl size=104650 sha256=4755d06fdc8ae8af63fbb6676c3a89eac4e0799cfbed2a9d5855db0be128aa44\n",
            "  Stored in directory: /root/.cache/pip/wheels/45/99/d7/c424029bb0fe910c63b68dbf2aa20d3283d023042521bcd7d5\n",
            "  Building wheel for pathtools (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pathtools: filename=pathtools-0.1.2-cp36-none-any.whl size=8784 sha256=41ad6461d19eed13a71ce9d1da052314a1757522f778a5cdee5fa0709ccecb96\n",
            "  Stored in directory: /root/.cache/pip/wheels/0b/04/79/c3b0c3a0266a3cb4376da31e5bfe8bba0c489246968a68e843\n",
            "  Building wheel for psutil (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for psutil: filename=psutil-5.7.0-cp36-cp36m-linux_x86_64.whl size=272670 sha256=f963f8761118ff8ddb5387ef375ffb61e5431603c43cf9b40a2836650aa6f5c6\n",
            "  Stored in directory: /root/.cache/pip/wheels/d7/69/b4/3200b95828d1f0ddb3cb5699083717f4fdbd9b4223d0644c57\n",
            "  Building wheel for PyYAML (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for PyYAML: filename=PyYAML-5.3.1-cp36-cp36m-linux_x86_64.whl size=44621 sha256=f720be0a73f8a4f7adb75c150e85df89054cb16870efd2cc6168da3e017c9ac3\n",
            "  Stored in directory: /root/.cache/pip/wheels/a7/c1/ea/cf5bd31012e735dc1dfea3131a2d5eae7978b251083d6247bd\n",
            "  Building wheel for subprocess32 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for subprocess32: filename=subprocess32-3.5.4-cp36-none-any.whl size=6489 sha256=bdfe39912cdb24cbb11306d0b55c9da965a10a70ce920e4443e2f281c09dbb7a\n",
            "  Stored in directory: /root/.cache/pip/wheels/68/39/1a/5e402bdfdf004af1786c8b853fd92f8c4a04f22aad179654d1\n",
            "  Building wheel for watchdog (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for watchdog: filename=watchdog-0.10.2-cp36-none-any.whl size=73605 sha256=481cfd09bf33994cc8f61429cee34f204f5473ecaa29c12a46bebc9ce934df23\n",
            "  Stored in directory: /root/.cache/pip/wheels/bc/ed/6c/028dea90d31b359cd2a7c8b0da4db80e41d24a59614154072e\n",
            "Successfully built future gql graphql-core pathtools psutil PyYAML subprocess32 watchdog\n",
            "\u001b[31mERROR: tensorflow 2.2.0 has requirement scipy==1.4.1; python_version >= \"3\", but you'll have scipy 1.5.0 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: kaggle 1.5.6 has requirement urllib3<1.25,>=1.21.1, but you'll have urllib3 1.25.9 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: google-colab 1.0.0 has requirement google-auth~=1.17.2, but you'll have google-auth 1.18.0 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: google-colab 1.0.0 has requirement requests~=2.23.0, but you'll have requests 2.24.0 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: google-colab 1.0.0 has requirement six~=1.12.0, but you'll have six 1.15.0 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: datascience 0.10.6 has requirement folium==0.2.1, but you'll have folium 0.8.3 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: albumentations 0.1.12 has requirement imgaug<0.2.7,>=0.2.5, but you'll have imgaug 0.2.9 which is incompatible.\u001b[0m\n",
            "Installing collected packages: appdirs, regex, typed-ast, toml, pathspec, black, cachetools, configparser, diskcache, six, docker-pycreds, mccabe, pyflakes, pycodestyle, importlib-metadata, flake8, future, smmap, gitdb, GitPython, google-auth, graphql-core, idna, urllib3, requests, gql, joblib, lxml, numpy, pathtools, psutil, tqdm, PyYAML, pytorch-lightning, pytz, threadpoolctl, scipy, scikit-learn, sentry-sdk, shortuuid, thinc, spacy, subprocess32, tensorboard-plugin-wit, watchdog, wandb, somajo\n",
            "  Found existing installation: regex 2019.12.20\n",
            "    Uninstalling regex-2019.12.20:\n",
            "      Successfully uninstalled regex-2019.12.20\n",
            "  Found existing installation: cachetools 4.1.1\n",
            "    Uninstalling cachetools-4.1.1:\n",
            "      Successfully uninstalled cachetools-4.1.1\n",
            "  Found existing installation: six 1.12.0\n",
            "    Uninstalling six-1.12.0:\n",
            "      Successfully uninstalled six-1.12.0\n",
            "  Found existing installation: importlib-metadata 1.7.0\n",
            "    Uninstalling importlib-metadata-1.7.0:\n",
            "      Successfully uninstalled importlib-metadata-1.7.0\n",
            "  Found existing installation: future 0.16.0\n",
            "    Uninstalling future-0.16.0:\n",
            "      Successfully uninstalled future-0.16.0\n",
            "  Found existing installation: google-auth 1.17.2\n",
            "    Uninstalling google-auth-1.17.2:\n",
            "      Successfully uninstalled google-auth-1.17.2\n",
            "  Found existing installation: idna 2.10\n",
            "    Uninstalling idna-2.10:\n",
            "      Successfully uninstalled idna-2.10\n",
            "  Found existing installation: urllib3 1.24.3\n",
            "    Uninstalling urllib3-1.24.3:\n",
            "      Successfully uninstalled urllib3-1.24.3\n",
            "  Found existing installation: requests 2.23.0\n",
            "    Uninstalling requests-2.23.0:\n",
            "      Successfully uninstalled requests-2.23.0\n",
            "  Found existing installation: joblib 0.16.0\n",
            "    Uninstalling joblib-0.16.0:\n",
            "      Successfully uninstalled joblib-0.16.0\n",
            "  Found existing installation: lxml 4.2.6\n",
            "    Uninstalling lxml-4.2.6:\n",
            "      Successfully uninstalled lxml-4.2.6\n",
            "  Found existing installation: numpy 1.18.5\n",
            "    Uninstalling numpy-1.18.5:\n",
            "      Successfully uninstalled numpy-1.18.5\n",
            "  Found existing installation: psutil 5.4.8\n",
            "    Uninstalling psutil-5.4.8:\n",
            "      Successfully uninstalled psutil-5.4.8\n",
            "  Found existing installation: tqdm 4.41.1\n",
            "    Uninstalling tqdm-4.41.1:\n",
            "      Successfully uninstalled tqdm-4.41.1\n",
            "  Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "  Found existing installation: pytz 2018.9\n",
            "    Uninstalling pytz-2018.9:\n",
            "      Successfully uninstalled pytz-2018.9\n",
            "  Found existing installation: scipy 1.4.1\n",
            "    Uninstalling scipy-1.4.1:\n",
            "      Successfully uninstalled scipy-1.4.1\n",
            "  Found existing installation: scikit-learn 0.22.2.post1\n",
            "    Uninstalling scikit-learn-0.22.2.post1:\n",
            "      Successfully uninstalled scikit-learn-0.22.2.post1\n",
            "  Found existing installation: thinc 7.4.0\n",
            "    Uninstalling thinc-7.4.0:\n",
            "      Successfully uninstalled thinc-7.4.0\n",
            "  Found existing installation: spacy 2.2.4\n",
            "    Uninstalling spacy-2.2.4:\n",
            "      Successfully uninstalled spacy-2.2.4\n",
            "  Found existing installation: tensorboard-plugin-wit 1.7.0\n",
            "    Uninstalling tensorboard-plugin-wit-1.7.0:\n",
            "      Successfully uninstalled tensorboard-plugin-wit-1.7.0\n",
            "Successfully installed GitPython-3.1.3 PyYAML-5.3.1 appdirs-1.4.4 black-19.10b0 cachetools-4.1.0 configparser-5.0.0 diskcache-4.1.0 docker-pycreds-0.4.0 flake8-3.8.3 future-0.18.2 gitdb-4.0.5 google-auth-1.18.0 gql-0.2.0 graphql-core-1.1 idna-2.9 importlib-metadata-1.6.1 joblib-0.15.1 lxml-4.5.1 mccabe-0.6.1 numpy-1.19.0 pathspec-0.8.0 pathtools-0.1.2 psutil-5.7.0 pycodestyle-2.6.0 pyflakes-2.2.0 pytorch-lightning-0.8.1 pytz-2020.1 regex-2020.6.8 requests-2.24.0 scikit-learn-0.23.1 scipy-1.5.0 sentry-sdk-0.15.1 shortuuid-1.0.1 six-1.15.0 smmap-3.0.4 somajo-2.1.1 spacy-2.3.0 subprocess32-3.5.4 tensorboard-plugin-wit-1.6.0.post3 thinc-7.4.1 threadpoolctl-2.1.0 toml-0.10.1 tqdm-4.46.1 typed-ast-1.4.1 urllib3-1.25.9 wandb-0.9.1 watchdog-0.10.2\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "cachetools",
                  "google",
                  "idna",
                  "numpy",
                  "pytz",
                  "requests",
                  "six",
                  "tqdm",
                  "urllib3"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GOSwh60bY6Eg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 573
        },
        "outputId": "07f3aaba-e67d-4a97-9348-9d04f52b4221",
        "tags": []
      },
      "source": [
        "!wget -O \"raw.de.gz\" http://opus.nlpl.eu/download.php?f=OpenSubtitles/v2018/mono/OpenSubtitles.raw.de.gz\n",
        "!gunzip \"raw.de.gz\"\n",
        "\n",
        "!wget -O \"raw.en.gz\" http://opus.nlpl.eu/download.php?f=OpenSubtitles/v2018/mono/OpenSubtitles.raw.en.gz\n",
        "!gunzip \"raw.en.gz\""
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": "--2020-09-21 20:36:06--  http://opus.nlpl.eu/download.php?f=OpenSubtitles/v2018/mono/OpenSubtitles.raw.de.gz\nResolving opus.nlpl.eu (opus.nlpl.eu)...193.166.25.9\nConnecting to opus.nlpl.eu (opus.nlpl.eu)|193.166.25.9|:80...connected.\nHTTP request sent, awaiting response...302 Found\nLocation: https://object.pouta.csc.fi/OPUS-OpenSubtitles/v2018/mono/de.txt.gz [following]\n--2020-09-21 20:36:06--  https://object.pouta.csc.fi/OPUS-OpenSubtitles/v2018/mono/de.txt.gz\nResolving object.pouta.csc.fi (object.pouta.csc.fi)...86.50.254.19, 86.50.254.18\nConnecting to object.pouta.csc.fi (object.pouta.csc.fi)|86.50.254.19|:443...connected.\nHTTP request sent, awaiting response...200 OK\nLength: 475884373 (454M) [application/gzip]\nSaving to: ‘raw.de.gz’\n\nraw.de.gz           100%[===================>] 453.84M  2.10MB/s    in 3m 23s  \n\n2020-09-21 20:39:30 (2.23 MB/s) - ‘raw.de.gz’ saved [475884373/475884373]\n\n--2020-09-21 20:39:38--  http://opus.nlpl.eu/download.php?f=OpenSubtitles/v2018/mono/OpenSubtitles.raw.en.gz\nResolving opus.nlpl.eu (opus.nlpl.eu)... 193.166.25.9\nConnecting to opus.nlpl.eu (opus.nlpl.eu)|193.166.25.9|:80...connected.\nHTTP request sent, awaiting response...302 Found\nLocation: https://object.pouta.csc.fi/OPUS-OpenSubtitles/v2018/mono/en.txt.gz [following]\n--2020-09-21 20:39:38--  https://object.pouta.csc.fi/OPUS-OpenSubtitles/v2018/mono/en.txt.gz\nResolving object.pouta.csc.fi (object.pouta.csc.fi)...86.50.254.18, 86.50.254.19\nConnecting to object.pouta.csc.fi (object.pouta.csc.fi)|86.50.254.18|:443...connected.\nHTTP request sent, awaiting response...200 OK\nLength: 3663376519 (3.4G) [application/gzip]\nSaving to: ‘raw.en.gz’\n\nraw.en.gz           100%[===================>]   3.41G  2.27MB/s    in 27m 24s \n\n2020-09-21 21:07:03 (2.12 MB/s) - ‘raw.en.gz’ saved [3663376519/3663376519]\n\n"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2FPsl_g7e8WQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "8adda155-7504-432e-cbc9-b2a4347e1164",
        "tags": []
      },
      "source": [
        "!python -m spacy download de_core_news_sm\n",
        "!python -m spacy download en_core_web_sm"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": "Requirement already satisfied: de_core_news_sm==2.3.0 from https://github.com/explosion/spacy-models/releases/download/de_core_news_sm-2.3.0/de_core_news_sm-2.3.0.tar.gz#egg=de_core_news_sm==2.3.0 in /home/bminixhofer/base/lib/python3.6/site-packages (2.3.0)\nRequirement already satisfied: spacy<2.4.0,>=2.3.0 in /home/bminixhofer/base/lib/python3.6/site-packages (from de_core_news_sm==2.3.0) (2.3.0)\nRequirement already satisfied: preshed<3.1.0,>=3.0.2 in /home/bminixhofer/base/lib/python3.6/site-packages (from spacy<2.4.0,>=2.3.0->de_core_news_sm==2.3.0) (3.0.2)\nRequirement already satisfied: tqdm<5.0.0,>=4.38.0 in /home/bminixhofer/base/lib/python3.6/site-packages (from spacy<2.4.0,>=2.3.0->de_core_news_sm==2.3.0) (4.46.1)\nRequirement already satisfied: setuptools in /home/bminixhofer/base/lib/python3.6/site-packages (from spacy<2.4.0,>=2.3.0->de_core_news_sm==2.3.0) (50.3.0)\nRequirement already satisfied: numpy>=1.15.0 in /home/bminixhofer/base/lib/python3.6/site-packages (from spacy<2.4.0,>=2.3.0->de_core_news_sm==2.3.0) (1.19.0)\nRequirement already satisfied: requests<3.0.0,>=2.13.0 in /home/bminixhofer/base/lib/python3.6/site-packages (from spacy<2.4.0,>=2.3.0->de_core_news_sm==2.3.0) (2.24.0)\nRequirement already satisfied: srsly<1.1.0,>=1.0.2 in /home/bminixhofer/base/lib/python3.6/site-packages (from spacy<2.4.0,>=2.3.0->de_core_news_sm==2.3.0) (1.0.2)\nRequirement already satisfied: blis<0.5.0,>=0.4.0 in /home/bminixhofer/base/lib/python3.6/site-packages (from spacy<2.4.0,>=2.3.0->de_core_news_sm==2.3.0) (0.4.1)\nRequirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /home/bminixhofer/base/lib/python3.6/site-packages (from spacy<2.4.0,>=2.3.0->de_core_news_sm==2.3.0) (1.0.2)\nRequirement already satisfied: thinc==7.4.1 in /home/bminixhofer/base/lib/python3.6/site-packages (from spacy<2.4.0,>=2.3.0->de_core_news_sm==2.3.0) (7.4.1)\nRequirement already satisfied: wasabi<1.1.0,>=0.4.0 in /home/bminixhofer/base/lib/python3.6/site-packages (from spacy<2.4.0,>=2.3.0->de_core_news_sm==2.3.0) (0.7.0)\nRequirement already satisfied: catalogue<1.1.0,>=0.0.7 in /home/bminixhofer/base/lib/python3.6/site-packages (from spacy<2.4.0,>=2.3.0->de_core_news_sm==2.3.0) (1.0.0)\nRequirement already satisfied: cymem<2.1.0,>=2.0.2 in /home/bminixhofer/base/lib/python3.6/site-packages (from spacy<2.4.0,>=2.3.0->de_core_news_sm==2.3.0) (2.0.3)\nRequirement already satisfied: plac<1.2.0,>=0.9.6 in /home/bminixhofer/base/lib/python3.6/site-packages (from spacy<2.4.0,>=2.3.0->de_core_news_sm==2.3.0) (1.1.3)\nRequirement already satisfied: certifi>=2017.4.17 in /home/bminixhofer/base/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy<2.4.0,>=2.3.0->de_core_news_sm==2.3.0) (2020.6.20)\nRequirement already satisfied: chardet<4,>=3.0.2 in /home/bminixhofer/base/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy<2.4.0,>=2.3.0->de_core_news_sm==2.3.0) (3.0.4)\nRequirement already satisfied: idna<3,>=2.5 in /home/bminixhofer/base/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy<2.4.0,>=2.3.0->de_core_news_sm==2.3.0) (2.9)\nRequirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /home/bminixhofer/base/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy<2.4.0,>=2.3.0->de_core_news_sm==2.3.0) (1.25.9)\nRequirement already satisfied: importlib-metadata>=0.20; python_version < \"3.8\" in /home/bminixhofer/base/lib/python3.6/site-packages (from catalogue<1.1.0,>=0.0.7->spacy<2.4.0,>=2.3.0->de_core_news_sm==2.3.0) (1.6.1)\nRequirement already satisfied: zipp>=0.5 in /home/bminixhofer/base/lib/python3.6/site-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->catalogue<1.1.0,>=0.0.7->spacy<2.4.0,>=2.3.0->de_core_news_sm==2.3.0) (3.1.0)\n\u001b[38;5;2m✔ Download and installation successful\u001b[0m\nYou can now load the model via spacy.load('de_core_news_sm')\nCollecting en_core_web_sm==2.3.0\n  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.3.0/en_core_web_sm-2.3.0.tar.gz (12.0 MB)\n\u001b[K     |████████████████████████████████| 12.0 MB 2.3 MB/s \n\u001b[?25hRequirement already satisfied: spacy<2.4.0,>=2.3.0 in /home/bminixhofer/base/lib/python3.6/site-packages (from en_core_web_sm==2.3.0) (2.3.0)\nRequirement already satisfied: preshed<3.1.0,>=3.0.2 in /home/bminixhofer/base/lib/python3.6/site-packages (from spacy<2.4.0,>=2.3.0->en_core_web_sm==2.3.0) (3.0.2)\nRequirement already satisfied: cymem<2.1.0,>=2.0.2 in /home/bminixhofer/base/lib/python3.6/site-packages (from spacy<2.4.0,>=2.3.0->en_core_web_sm==2.3.0) (2.0.3)\nRequirement already satisfied: tqdm<5.0.0,>=4.38.0 in /home/bminixhofer/base/lib/python3.6/site-packages (from spacy<2.4.0,>=2.3.0->en_core_web_sm==2.3.0) (4.46.1)\nRequirement already satisfied: requests<3.0.0,>=2.13.0 in /home/bminixhofer/base/lib/python3.6/site-packages (from spacy<2.4.0,>=2.3.0->en_core_web_sm==2.3.0) (2.24.0)\nRequirement already satisfied: srsly<1.1.0,>=1.0.2 in /home/bminixhofer/base/lib/python3.6/site-packages (from spacy<2.4.0,>=2.3.0->en_core_web_sm==2.3.0) (1.0.2)\nRequirement already satisfied: catalogue<1.1.0,>=0.0.7 in /home/bminixhofer/base/lib/python3.6/site-packages (from spacy<2.4.0,>=2.3.0->en_core_web_sm==2.3.0) (1.0.0)\nRequirement already satisfied: plac<1.2.0,>=0.9.6 in /home/bminixhofer/base/lib/python3.6/site-packages (from spacy<2.4.0,>=2.3.0->en_core_web_sm==2.3.0) (1.1.3)\nRequirement already satisfied: thinc==7.4.1 in /home/bminixhofer/base/lib/python3.6/site-packages (from spacy<2.4.0,>=2.3.0->en_core_web_sm==2.3.0) (7.4.1)\nRequirement already satisfied: setuptools in /home/bminixhofer/base/lib/python3.6/site-packages (from spacy<2.4.0,>=2.3.0->en_core_web_sm==2.3.0) (50.3.0)\nRequirement already satisfied: numpy>=1.15.0 in /home/bminixhofer/base/lib/python3.6/site-packages (from spacy<2.4.0,>=2.3.0->en_core_web_sm==2.3.0) (1.19.0)\nRequirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /home/bminixhofer/base/lib/python3.6/site-packages (from spacy<2.4.0,>=2.3.0->en_core_web_sm==2.3.0) (1.0.2)\nRequirement already satisfied: blis<0.5.0,>=0.4.0 in /home/bminixhofer/base/lib/python3.6/site-packages (from spacy<2.4.0,>=2.3.0->en_core_web_sm==2.3.0) (0.4.1)\nRequirement already satisfied: wasabi<1.1.0,>=0.4.0 in /home/bminixhofer/base/lib/python3.6/site-packages (from spacy<2.4.0,>=2.3.0->en_core_web_sm==2.3.0) (0.7.0)\nRequirement already satisfied: certifi>=2017.4.17 in /home/bminixhofer/base/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy<2.4.0,>=2.3.0->en_core_web_sm==2.3.0) (2020.6.20)\nRequirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /home/bminixhofer/base/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy<2.4.0,>=2.3.0->en_core_web_sm==2.3.0) (1.25.9)\nRequirement already satisfied: chardet<4,>=3.0.2 in /home/bminixhofer/base/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy<2.4.0,>=2.3.0->en_core_web_sm==2.3.0) (3.0.4)\nRequirement already satisfied: idna<3,>=2.5 in /home/bminixhofer/base/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy<2.4.0,>=2.3.0->en_core_web_sm==2.3.0) (2.9)\nRequirement already satisfied: importlib-metadata>=0.20; python_version < \"3.8\" in /home/bminixhofer/base/lib/python3.6/site-packages (from catalogue<1.1.0,>=0.0.7->spacy<2.4.0,>=2.3.0->en_core_web_sm==2.3.0) (1.6.1)\nRequirement already satisfied: zipp>=0.5 in /home/bminixhofer/base/lib/python3.6/site-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->catalogue<1.1.0,>=0.0.7->spacy<2.4.0,>=2.3.0->en_core_web_sm==2.3.0) (3.1.0)\nBuilding wheels for collected packages: en-core-web-sm\n  Building wheel for en-core-web-sm (setup.py) ... \u001b[?25ldone\n\u001b[?25h  Created wheel for en-core-web-sm: filename=en_core_web_sm-2.3.0-py3-none-any.whl size=12048606 sha256=6e883750227d0cfd981fad1744736468d1da5c4b177af0797cf1b398bc2cca43\n  Stored in directory: /tmp/pip-ephem-wheel-cache-v276g5p_/wheels/4a/db/07/94eee4f3a60150464a04160bd0dfe9c8752ab981fe92f16aea\nSuccessfully built en-core-web-sm\nInstalling collected packages: en-core-web-sm\n  Attempting uninstall: en-core-web-sm\n    Found existing installation: en-core-web-sm 2.2.5\n    Uninstalling en-core-web-sm-2.2.5:\n      Successfully uninstalled en-core-web-sm-2.2.5\nSuccessfully installed en-core-web-sm-2.3.0\n\u001b[38;5;2m✔ Download and installation successful\u001b[0m\nYou can now load the model via spacy.load('en_core_web_sm')\n"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iU4MY_3bgUfP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import sys"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LAaerLgPis7e",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 176
        },
        "outputId": "0782c151-b5e8-4f8c-89da-8b8dfedd1e38",
        "tags": []
      },
      "source": [
        "!pip install nnsplit"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": "Collecting nnsplit\n  Downloading nnsplit-0.4.10_post0-cp36-cp36m-manylinux1_x86_64.whl (1.8 MB)\n\u001b[K     |████████████████████████████████| 1.8 MB 2.0 MB/s \n\u001b[?25hRequirement already satisfied, skipping upgrade: onnxruntime<2,>=1.4 in /home/bminixhofer/base/lib/python3.6/site-packages (from nnsplit) (1.4.0)\nRequirement already satisfied, skipping upgrade: numpy>=1.16.6 in /home/bminixhofer/base/lib/python3.6/site-packages (from onnxruntime<2,>=1.4->nnsplit) (1.19.0)\nRequirement already satisfied, skipping upgrade: protobuf in /home/bminixhofer/base/lib/python3.6/site-packages (from onnxruntime<2,>=1.4->nnsplit) (3.12.2)\nRequirement already satisfied, skipping upgrade: six>=1.9 in /home/bminixhofer/base/lib/python3.6/site-packages (from protobuf->onnxruntime<2,>=1.4->nnsplit) (1.15.0)\nRequirement already satisfied, skipping upgrade: setuptools in /home/bminixhofer/base/lib/python3.6/site-packages (from protobuf->onnxruntime<2,>=1.4->nnsplit) (50.3.0)\nInstalling collected packages: nnsplit\n  Attempting uninstall: nnsplit\n    Found existing installation: nnsplit 0.4.9.post0\n    Uninstalling nnsplit-0.4.9.post0:\n      Successfully uninstalled nnsplit-0.4.9.post0\nSuccessfully installed nnsplit-0.4.10.post0\n"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aFjfHdotXwSn",
        "colab_type": "text"
      },
      "source": [
        "# Evaluate"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hiab-PklW7F4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import sys\n",
        "sys.path.append(\"nnsplit/train\")"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TZPcLRZ1W_tZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import evaluate\n",
        "from evaluate import OpenSubtitlesDataset, Evaluator\n",
        "from torch.utils.data import Subset\n",
        "from nnsplit import NNSplit\n",
        "import spacy\n",
        "import numpy as np\n",
        "import torch\n",
        "import pandas as pd"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uBXD-JUubxNU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class NNSplitInterface:\n",
        "  def __init__(self, splitter):\n",
        "    self.splitter = splitter\n",
        "\n",
        "  def split(self, texts):\n",
        "    out = []\n",
        "    for split in self.splitter.split(texts):\n",
        "        out.append([str(x) for x in split])\n",
        "\n",
        "    return out\n",
        "  \n",
        "class SpacyInterface:\n",
        "  def __init__(self, name, use_sentencizer):\n",
        "    if use_sentencizer:\n",
        "      nlp = spacy.load(name, disable=[\"tagger\", \"parser\", \"ner\"])\n",
        "      nlp.add_pipe(nlp.create_pipe(\"sentencizer\"))\n",
        "    else:\n",
        "      nlp = spacy.load(name, disable=[\"tagger\", \"ner\"])\n",
        "\n",
        "    self.nlp = nlp\n",
        "\n",
        "  def split(self, texts):\n",
        "    out = []\n",
        "    for doc in self.nlp.pipe(texts):\n",
        "        sentences = []\n",
        "\n",
        "        for sent in doc.sents:\n",
        "            sentences.append(\"\".join([x.text + x.whitespace_ for x in sent]))\n",
        "\n",
        "        out.append(sentences)\n",
        "\n",
        "    return out"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x2lBbStLbRsJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 116,
          "referenced_widgets": [
            "c582702a4f1d4528a76e30299f599dfb",
            "930c4a209a844140a8c63622eafdd8f0",
            "654b5194415b4b6482b067a57136b63a",
            "d8eb5a2f52a046b0817f28b52d7daefc",
            "963e890484e3409dbc7bb888866ab429",
            "60296b4a87eb4fb8b2dc8c0150ccd5d9",
            "91a188a8b8234c17a085e062542921a6",
            "61dd7b2a28884b64a238b6c723d1fbe6",
            "38742876bb934c91a82a5e48c0e395d6",
            "6568e9f2bc1a495cab978eb914c4eac4",
            "6f337f837a124f449a554e768812ef9c",
            "66023a2e82ac4bfa9141948ad0080ed9",
            "fbd9984494214436a14d802e8643d105",
            "a82576995afe4ef1bbb0ff49498466c4",
            "0b27d3de82294159a6cd675777df9e89",
            "967cdcb361f044fb9ce43a671ba6e578"
          ]
        },
        "outputId": "35b4f08d-ca88-4895-b643-5d1c3a71b04b",
        "tags": []
      },
      "source": [
        "data = [\n",
        "  [\n",
        "   \"German\",\n",
        "   Subset(OpenSubtitlesDataset(\"../train_data/de/raw.de\", 1_000_000), np.arange(100_000)),\n",
        "   {\n",
        "       \"NNSplit\": NNSplitInterface(NNSplit(\"models/german/model.onnx\")),\n",
        "       \"Spacy (Tagger)\": SpacyInterface(\"de_core_news_sm\", use_sentencizer=False),\n",
        "       \"Spacy (Sentencizer)\": SpacyInterface(\"de_core_news_sm\", use_sentencizer=True)\n",
        "   }\n",
        "  ],\n",
        "]\n",
        "\n",
        "eval_setups = {\n",
        "    \"Clean\": (0.0, 0.0),\n",
        "    \"Partial punctuation\": (0.5, 0.0),\n",
        "    \"Partial case\": (0.0, 0.5),\n",
        "    \"Partial punctuation and case\": (0.5, 0.5),\n",
        "    \"No punctuation and case\": (1.0, 1.0),\n",
        "}"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e8dc2025b4d94a87acb40d902d7e2b5b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0jKDR28LXShq",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 337,
          "referenced_widgets": [
            "509f5bca62e8462eaa6b76447ddd1c4a",
            "8db4c8a02e26469aa4f3f4106ec6959e",
            "4a9de0bf801b49418ec379da9ccd0326",
            "0499a851aac0452dbc0160e5215a8739",
            "f20883f8897f4ac582486ab4106b6cb9",
            "ed47142bc02c46d59475c865e5db2f71",
            "a358b6fc3c3649709575c7dff023f39d",
            "465fc7dad2dd4561bd13a58578c49cc3",
            "46a77d630c064c46bdd54389cd747066",
            "44733acb73fe4c73a0956e0b96bba3a3",
            "7ebfbc2b458044298cc5a9ec145ed1b1",
            "843531c02ec54712b7018913f1780fc8",
            "df333f77dec041b79674dd0b39c3cabf",
            "cc6a48a04c2d4ca29d980828c310c0fc",
            "8cc4c9f2763946db98793945386a40ff",
            "03b884c6f5c14b5685c0076313096ed8",
            "79e43df3e5864f30a9cc0a9c0b76d126",
            "d5942bc52c4d4ababa468f798876a8bc",
            "c8f213e3b58f47c2bae93fb89d8cfd32",
            "e27116b1b18f4247af0b5e071a6d226e",
            "f9f17a90194b42eb8ae2c2f0c582b39a",
            "863d015cc1004e53b544432f3cac298a",
            "0ed78a1bab0646e39f16f7c71a630ec5",
            "93e8fc6344b74522bec775f006286975",
            "8574e7d9d8cd466681f86f62d7e6de65",
            "255b58b192024725a451a056be2d72d7",
            "6b88aab9f1a34d1f91a183e5688d72cd",
            "526aa70f8a8d4be78645aab166c00887",
            "8a934620dc94475cbb5841c392fa8537",
            "64fe9eec0df54e3c9c270b3edb9bde05",
            "31c1355fbe1a4ef7ab6da08ac9b7051b",
            "71187f09e69a42779152b3b499e5ab7c",
            "ab4e7df6377548a3bdef2332fa9ca8b4",
            "857925820b7c4064952d0565189b6a31",
            "2066e14366a047d39eda7ec0f1392105",
            "0204164bfd4840f49bfb446c7f5690f9",
            "75c17a4252b243d5ba6a6f613331dcd2",
            "8fbea136e5c149c7915ab08fd529e952",
            "4cd0ac2896a747aabe558b076d90f3e4",
            "695272b807bc4015a051f39bf45b45a3",
            "3440bc555d324c7bad6c2ee4c75ed674",
            "e0ef9ce972a64f5f8675a2ad5df891e7",
            "a4858133405a4534bf49ef43b5aeb65a",
            "387837219c8648f2a8b259e20c564408",
            "acd406ef16c7442d9cc2174cd4b81974",
            "b1ed323d762a4b77976937c517693cb0",
            "2e43010cbcec47d1960c72918397a81e",
            "02a8a6a7fb1c4fedbd0878ab0a0276bb",
            "7b49eb47c1754b4da877866c19059834",
            "63736e9e37044ffbb8e531959b2de060",
            "e20b17c5669048698267cf341add521f",
            "ef9acfcd8c3c40a5924fbf2691dadcf6",
            "3ff3243240b74103b4412da37a0badc3",
            "9aa6c0b352d24cce8a110ce167618bf4",
            "95798d01a09541218c176006cff5e0fa",
            "aae96f1bfb3a4a9087b77aecfc9747e1",
            "90023547667c4f3f80370d7e2baac5ea",
            "111c6d3fdcc7416c8f4ce17ccb59c8e8",
            "a65bc11ad7104be28c7ce3c5cc4c0766",
            "b3cb537f4ad745cbb3cf1ddc6c8974bf",
            "28c1e2fdcf404b04ab0d43644ae595d2",
            "43c9061a42784dda93d30f1e7b723ec7",
            "0ed0a5b4b7ae4bd08ee83afe5393095d",
            "44ed94b24c33412da95e20c563f39671",
            "1a41adb523f941dfb3140403b9a9f813",
            "2da0f6faf5cc429eb699c89b8465d358",
            "99102b4d5c3f41cebcc66b2ca53a65d6",
            "68c8c0bff4b145f485c61586b1ab0943",
            "0078481b48ca46cea0ceda01a917ce33",
            "0f0be53f3c0344d785257844220984f2",
            "23faa4279c654f7599e2743cf635f9f3",
            "8dfbae54e84c4323aceb94315e8e3fc5",
            "cb5b69e553e844c18e18a8a90c3fdf91",
            "1d598b2ad7f8402d8d1649d97c639949",
            "4fdbb0caa5794801ae4c3ee3219f6da6",
            "c94efc671d094f8fb788b42b570fc278",
            "f3572c0d3797458fa4946662244e6407",
            "4c7b4d004967432abc757360124b0f0d",
            "b1b47a50e27144e2b018f8ddd27d57db",
            "5c763d671d474583bb46635e0254cdab"
          ]
        },
        "outputId": "7c8bffc0-e07a-448c-fd36-6babeb0078d5",
        "tags": []
      },
      "source": [
        "results = {}\n",
        "preds = {}\n",
        "\n",
        "for dataset_name, dataset, targets in data:\n",
        "  results[dataset_name] = {}\n",
        "\n",
        "  for eval_name, (remove_punct_prob, lower_start_prob) in eval_setups.items():\n",
        "    results[dataset_name][eval_name] = {}\n",
        "    evaluator = Evaluator(dataset, remove_punct_prob, lower_start_prob)\n",
        "\n",
        "    for target_name, interface in targets.items():\n",
        "      correct = evaluator.evaluate(interface.split)\n",
        "      preds[f\"{dataset_name}_{eval_name}_{target_name}\"] = {\n",
        "          \"samples\": evaluator.texts,\n",
        "          \"correct\": correct,\n",
        "      }\n",
        "      results[dataset_name][eval_name][target_name] = correct.mean()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "HBox(children=(FloatProgress(value=0.0, max=100000.0), HTML(value='')))",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "019bfcc0e3804647a33266df35326284"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "HBox(children=(FloatProgress(value=0.0, max=100000.0), HTML(value='')))",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "627afbd7e5b3491484b9f6cafa18147a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "HBox(children=(FloatProgress(value=0.0, max=100000.0), HTML(value='')))",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bc933e6d1c1746f2a589b0a40eee88a2"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "HBox(children=(FloatProgress(value=0.0, max=100000.0), HTML(value='')))",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "603aa0fc93f844b8ab22f5485ed9a156"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "HBox(children=(FloatProgress(value=0.0, max=100000.0), HTML(value='')))",
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "88505d662a10403ea572e1db17963eb6"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {},
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                               NNSplit  Spacy (Tagger)  Spacy (Sentencizer)\n",
              "Clean                         0.795200        0.833368             0.878471\n",
              "Partial punctuation           0.531877        0.426458             0.266312\n",
              "Partial case                  0.803389        0.792839             0.876678\n",
              "Partial punctuation and case  0.526467        0.377201             0.266970\n",
              "No punctuation and case       0.293451        0.095227             0.007562"
            ],
            "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>NNSplit</th>\n      <th>Spacy (Tagger)</th>\n      <th>Spacy (Sentencizer)</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>Clean</th>\n      <td>0.795200</td>\n      <td>0.833368</td>\n      <td>0.878471</td>\n    </tr>\n    <tr>\n      <th>Partial punctuation</th>\n      <td>0.531877</td>\n      <td>0.426458</td>\n      <td>0.266312</td>\n    </tr>\n    <tr>\n      <th>Partial case</th>\n      <td>0.803389</td>\n      <td>0.792839</td>\n      <td>0.876678</td>\n    </tr>\n    <tr>\n      <th>Partial punctuation and case</th>\n      <td>0.526467</td>\n      <td>0.377201</td>\n      <td>0.266970</td>\n    </tr>\n    <tr>\n      <th>No punctuation and case</th>\n      <td>0.293451</td>\n      <td>0.095227</td>\n      <td>0.007562</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
          },
          "metadata": {},
          "execution_count": 7
        }
      ],
      "source": [
        "pd.DataFrame.from_dict(results[\"German\"]).T"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gdZt_YVr__wU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "40a19479-ed3a-4720-ff9d-b9fb984c9cca"
      },
      "source": [
        "pd.DataFrame.from_dict(results[\"English\"]).T"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>NNSplit</th>\n",
              "      <th>Spacy (Tagger)</th>\n",
              "      <th>Spacy (Sentencizer)</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Clean</th>\n",
              "      <td>0.754371</td>\n",
              "      <td>0.853603</td>\n",
              "      <td>0.820934</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Partial punctuation</th>\n",
              "      <td>0.485907</td>\n",
              "      <td>0.517829</td>\n",
              "      <td>0.249753</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Partial case</th>\n",
              "      <td>0.761754</td>\n",
              "      <td>0.825119</td>\n",
              "      <td>0.819679</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Partial punctuation and case</th>\n",
              "      <td>0.443704</td>\n",
              "      <td>0.458619</td>\n",
              "      <td>0.249873</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>No punctuation and case</th>\n",
              "      <td>0.166273</td>\n",
              "      <td>0.180859</td>\n",
              "      <td>0.004633</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                               NNSplit  Spacy (Tagger)  Spacy (Sentencizer)\n",
              "Clean                         0.754371        0.853603             0.820934\n",
              "Partial punctuation           0.485907        0.517829             0.249753\n",
              "Partial case                  0.761754        0.825119             0.819679\n",
              "Partial punctuation and case  0.443704        0.458619             0.249873\n",
              "No punctuation and case       0.166273        0.180859             0.004633"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    }
  ]
}